{
 "cells": [
  {
   "cell_type": "code",
   "id": "initial_id",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2024-05-10T03:10:41.683448Z",
     "start_time": "2024-05-10T03:10:41.420795Z"
    }
   },
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ],
   "execution_count": 2,
   "outputs": []
  },
  {
   "cell_type": "code",
   "id": "f0e721027d704047",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-10T03:10:42.593909Z",
     "start_time": "2024-05-10T03:10:42.559929Z"
    }
   },
   "source": [
    "data=pd.read_csv(\"./csv_dir/demo_result1.csv\")\n",
    "#| 预览前5行数据\n",
    "data.head()"
   ],
   "execution_count": 3,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "id": "8537eee19c833e95",
   "metadata": {},
   "source": [
    "### 异常数据检测"
   ]
  },
  {
   "cell_type": "code",
   "id": "ba401d34a1778102",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-10T03:10:44.877306Z",
     "start_time": "2024-05-10T03:10:44.869652Z"
    }
   },
   "source": [
    "data.isnull().sum()"
   ],
   "execution_count": 4,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "id": "9750cb63c9e51e73",
   "metadata": {},
   "source": [
    "### 数据预处理"
   ]
  },
  {
   "cell_type": "code",
   "id": "cdf4c001cfc5ab73",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-10T03:10:46.022375Z",
     "start_time": "2024-05-10T03:10:46.011162Z"
    }
   },
   "source": [
    "df_clean=data\n",
    "df_clean.info()\n",
    "df_clean.describe()"
   ],
   "execution_count": 5,
   "outputs": []
  },
  {
   "cell_type": "code",
   "id": "550174f3eb823c39",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-10T03:10:47.218585Z",
     "start_time": "2024-05-10T03:10:47.160499Z"
    }
   },
   "source": [
    "df_clean_copy = df_clean.copy()\n",
    "df_clean_copy['评论内容'] = df_clean_copy['评论内容'].str.replace(r'@[^\\s]+', '', regex=True)\n",
    "df_clean_copy['评论内容'] = df_clean_copy['评论内容'].str.replace(r'@', '')\n",
    "df_clean_copy['评论内容'] = df_clean_copy['评论内容'].str.replace(r'[', '')\n",
    "df_clean_copy['评论内容'] = df_clean_copy['评论内容'].str.replace(r']', '')\n",
    "df_clean_copy['评论内容'] = df_clean_copy['评论内容'].str.replace(r'\\s', '')\n",
    "df_clean_copy['评论内容'] = df_clean_copy['评论内容'].str.replace(r'“', '')\n",
    "df_clean_copy['评论内容'] = df_clean_copy['评论内容'].str.replace(r'”', '')\n",
    "df_clean_copy['评论内容'] = df_clean_copy['评论内容'].str.replace(r'。', '')\n",
    "df_clean_copy['评论内容'] = df_clean_copy['评论内容'].str.replace(r'\\n', '')\n",
    "df_clean_copy['评论内容'] = df_clean_copy['评论内容'].str.replace(r'\\t', '')\n",
    "df_clean_copy['评论内容'] = df_clean_copy['评论内容'].str.replace(r'，', '')\n",
    "df_clean_copy['评论内容'] = df_clean_copy['评论内容'].str.replace(r',', '')\n",
    "df_clean_copy['评论内容'] = df_clean_copy['评论内容'].str.replace(r'.', '')\n",
    "df_clean_copy['评论内容'] = df_clean_copy['评论内容'].str.replace(r'》', '')\n",
    "df_clean_copy['评论内容'] = df_clean_copy['评论内容'].str.replace(r'《', '')\n",
    "df_clean_copy['评论内容'] = df_clean_copy['评论内容'].str.replace(r'>', '')\n",
    "df_clean_copy['评论内容'] = df_clean_copy['评论内容'].str.replace(r'<', '')\n",
    "df_clean_copy['评论内容'] = df_clean_copy['评论内容'].str.replace(r' ', '')\n",
    "df_clean_copy['评论内容'] = df_clean_copy['评论内容'].str.replace(r'.', '')\n",
    "df_clean_copy['评论内容'] = df_clean_copy['评论内容'].str.replace(r'    ', '')\n",
    "df_clean_copy.replace(to_replace=r'^\\s*$', value=np.nan, regex=True, inplace=True)\n",
    "df_clean_copy = df_clean_copy.dropna(subset=['评论内容'], inplace=False)\n",
    "df_clean_copy.to_csv('./csv_dir/demo_result2.csv', mode='w',index=False,encoding='utf_8_sig')\n",
    "df_clean = df_clean_copy.copy()\n",
    "df_clean"
   ],
   "execution_count": 6,
   "outputs": []
  },
  {
   "cell_type": "code",
   "id": "a7fae61a86688e6f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-10T03:10:48.144349Z",
     "start_time": "2024-05-10T03:10:48.136548Z"
    }
   },
   "source": [
    "# df_clean.isnull().any()\n",
    "df_clean.isnull().sum()"
   ],
   "execution_count": 7,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "id": "b2b321fcccb23a8c",
   "metadata": {},
   "source": [
    "### 采用excel删除空白行，实在是累了"
   ]
  },
  {
   "cell_type": "code",
   "id": "71165a0a3772e1e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-10T03:10:49.297450Z",
     "start_time": "2024-05-10T03:10:49.274233Z"
    }
   },
   "source": [
    "test_data=pd.read_csv(\"./csv_dir/demo_result2.csv\").astype(str) \n",
    "# 预览前5行数据\n",
    "test_data.head()"
   ],
   "execution_count": 8,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "id": "7970f046f1c13754",
   "metadata": {},
   "source": [
    "### NLP"
   ]
  },
  {
   "cell_type": "code",
   "id": "9888fb8e8940ac79",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-09T15:15:14.333029Z",
     "start_time": "2024-05-09T15:15:13.411829Z"
    }
   },
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "# 初步构建词典，查看语料词典长度\n",
    "bow_vect = CountVectorizer(analyzer='word')\n",
    "bow_vect.fit(test_data['评论内容'])\n",
    "bow_vocab = bow_vect.get_feature_names_out() # 获取词典\n",
    "print('词典长度：', len(bow_vocab))"
   ],
   "execution_count": 13,
   "outputs": []
  },
  {
   "cell_type": "code",
   "id": "e35182c1d74004b3",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-09T15:15:15.012730Z",
     "start_time": "2024-05-09T15:15:14.976239Z"
    }
   },
   "source": [
    "%%time\n",
    "# 使用词袋表示作为文本的特征表示\n",
    "bow_vect = CountVectorizer(analyzer='word', max_features=5000)\n",
    "bow_features = bow_vect.fit_transform(test_data['评论内容'])  # 得到的是稀疏矩阵\n",
    "bow_features = bow_features.toarray()\n",
    "bow_vocab = bow_vect.get_feature_names_out()  # 获取词典\n",
    "print('词典长度：', len(bow_vocab))\n",
    "print('shape of bow representation:', bow_features.shape)"
   ],
   "execution_count": 14,
   "outputs": []
  },
  {
   "cell_type": "code",
   "id": "59dd4ea038f404ff",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-09T15:15:15.786198Z",
     "start_time": "2024-05-09T15:15:15.756929Z"
    }
   },
   "source": [
    "%%time\n",
    "# 使用tfidf权重作为文本的特征表示\n",
    "tfidf_vect = TfidfVectorizer(max_features=5000)\n",
    "tfidf_features = tfidf_vect.fit_transform(test_data['评论内容'])\n",
    "tfidf_vocab = tfidf_vect.get_feature_names_out()  # 获取词典\n",
    "print('词典长度：', len(tfidf_vocab))\n",
    "print('shape of tfidf representation:', tfidf_features.shape)"
   ],
   "execution_count": 15,
   "outputs": []
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "### 制作词云图",
   "id": "cd7e24c5debd925d"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-09T15:15:17.938833Z",
     "start_time": "2024-05-09T15:15:16.746912Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from jieba  import analyse\n",
    "import jieba\n",
    "# 引入TF-IDF关键词抽取接口\n",
    "tfidf = analyse.extract_tags\n",
    "\n",
    "\n",
    "# 原始文本\n",
    "text = ('谁说不是呐超爱❤️送心我的天谁懂他的含金量那种氛围感啊真的好帅谁懂谁懂啊啊啊啊啊反正我是懂好听奸笑而/'\n",
    "        '且不止唱歌好听还很有耐心经常骑老奶奶过马路比心比心比心比心')\n",
    " \n",
    "# 基于TF-IDF算法进行关键词抽取\n",
    "'''\n",
    "sentence 为待提取的文本\n",
    "topK 为返回几个 TF/IDF 权重最大的关键词，默认值为 20\n",
    "withWeight 为是否一并返回关键词权重值，默认值为 False\n",
    "allowPOS 仅包括指定词性的词，默认值为空，即不筛选\n",
    "'''\n",
    "\n",
    "result = test_data['评论内容'].str.join('')\n",
    "result = result.to_string(index=False)  \n",
    "result.replace(\" \",\"\")\n",
    "\n",
    "keywords = tfidf(result,topK=50, withWeight=False, allowPOS=())\n",
    "\n",
    "\n",
    "keywords = ','.join(keywords)\n",
    "print(keywords)\n",
    "print(\"完成\")"
   ],
   "id": "bdaa88740f0d6610",
   "execution_count": 16,
   "outputs": []
  },
  {
   "cell_type": "code",
   "id": "7ab9099010b2f139",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-09T15:15:19.712437Z",
     "start_time": "2024-05-09T15:15:19.221690Z"
    }
   },
   "source": [
    "from wordcloud import WordCloud\n",
    "from PIL import Image\n",
    "\n",
    "wcd=WordCloud(background_color='white', max_words=50, repeat=True,max_font_size=100,font_path='./fonts/STXINGKA.TTF').generate(keywords)\n",
    "wcd.to_image()"
   ],
   "execution_count": 17,
   "outputs": []
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-09T04:15:59.312406Z",
     "start_time": "2024-05-09T04:15:59.018699Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from wordcloud import WordCloud\n",
    "from PIL import Image\n",
    "wcd=WordCloud(background_color='white', max_words=50, repeat=True,max_font_size=100,font_path='./fonts/STXINGKA.TTF',height=600,width=800).generate(keywords)\n",
    "wcd.to_image()"
   ],
   "id": "a376abc8326bf8b2",
   "execution_count": 55,
   "outputs": []
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "### 黑子白子比例，snownlp",
   "id": "9df6679814aa890b"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-09T15:17:26.288285Z",
     "start_time": "2024-05-09T15:17:14.050040Z"
    }
   },
   "cell_type": "code",
   "source": [
    "provinces = []\n",
    "# 创建字典，键为省份名称（不包含“省”），值为0\n",
    "province_dict = {province: 0 for province in provinces}\n",
    "province_dict2 = {province: 0 for province in provinces}\n",
    "# snownlp适合电商其实，不适合做网络热词评论的情绪分析\n",
    "from snownlp import SnowNLP\n",
    "withe_count=0\n",
    "withe_dict=province_dict\n",
    "black_count=0\n",
    "black_dict=province_dict2\n",
    "for index,word in test_data.iterrows():\n",
    "    if word['评论IP属地'] not in withe_dict:   \n",
    "        withe_dict[word['评论IP属地']]=0\n",
    "    if word['评论IP属地'] not in black_dict:   \n",
    "        black_dict[word['评论IP属地']]=0\n",
    "    s = SnowNLP(word['评论内容'])\n",
    "    sentiments = s.sentiments\n",
    "    #print(sentiments)\n",
    "    if sentiments>0.3:\n",
    "        #print(\"白子\")\n",
    "        withe_dict[str(word['评论IP属地'])]+=1\n",
    "        withe_count=withe_count+1\n",
    "    else:\n",
    "        #print(\"黑子\")\n",
    "        black_dict[str(word['评论IP属地'])]+=1\n",
    "        black_count=black_count+1\n",
    "  \n",
    "        \n",
    "print(withe_count,black_count)\n",
    "df = pd.DataFrame(list(withe_dict.items()), columns=['ip', 'count'])\n",
    "df2 = pd.DataFrame(list(black_dict.items()), columns=['ip', 'count'])\n",
    "df\n",
    "#black_dict = pd.DataFrame(black_dict)"
   ],
   "id": "d1c1bd5de43dd8e1",
   "execution_count": 19,
   "outputs": []
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-09T15:17:30.563948Z",
     "start_time": "2024-05-09T15:17:30.560590Z"
    }
   },
   "cell_type": "code",
   "source": "df['count'].sum(),df2['count'].sum()",
   "id": "7b47c1e80873f066",
   "execution_count": 20,
   "outputs": []
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-09T15:17:32.334076Z",
     "start_time": "2024-05-09T15:17:32.156861Z"
    }
   },
   "cell_type": "code",
   "source": [
    "df = df[df['count'] > 30]\n",
    "# 按值的大小对数据进行降序排序\n",
    "df = df.sort_values(by='count', ascending=False)\n",
    "plt.rcParams['font.sans-serif'] = ['SimHei']  # 指定默认字体\n",
    "plt.rcParams['axes.unicode_minus'] = False  # 解决保存图像时负号'-'显示为方块的问题\n",
    "\n",
    "plt.figure(figsize=(10, 10))\n",
    "\n",
    "plt.pie(df['count'], labels=df['ip'], autopct='%1.1f%%',startangle=140)\n",
    "# 将饼图的中心设置为图中心，使标签在饼图外部\n",
    "centre_circle = plt.Circle((0,0),0.90,fc='white')\n",
    "fig = plt.gcf()\n",
    "fig.gca().add_artist(centre_circle)\n",
    "plt.title('白子所属ip饼图')\n",
    "plt.show()"
   ],
   "id": "f809981043355de",
   "execution_count": 21,
   "outputs": []
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-09T15:17:34.187420Z",
     "start_time": "2024-05-09T15:17:34.071353Z"
    }
   },
   "cell_type": "code",
   "source": [
    "df2 = df2[df2['count'] > 30]\n",
    "# 按值的大小对数据进行降序排序\n",
    "df = df.sort_values(by='count', ascending=False)\n",
    "plt.rcParams['font.sans-serif'] = ['SimHei']  # 指定默认字体\n",
    "plt.rcParams['axes.unicode_minus'] = False  # 解决保存图像时负号'-'显示为方块的问题\n",
    "\n",
    "plt.figure(figsize=(10, 10))\n",
    "\n",
    "plt.pie(df2['count'], labels=df2['ip'], autopct='%1.1f%%',startangle=140)\n",
    "# 将饼图的中心设置为图中心，使标签在饼图外部\n",
    "centre_circle = plt.Circle((0,0),0.90,fc='white')\n",
    "fig = plt.gcf()\n",
    "fig.gca().add_artist(centre_circle)\n",
    "plt.title('黑子所属ip饼图')\n",
    "plt.show()"
   ],
   "id": "5b4e0966c8bca9a5",
   "execution_count": 22,
   "outputs": []
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-10T05:27:10.781836Z",
     "start_time": "2024-05-10T05:27:10.773524Z"
    }
   },
   "cell_type": "code",
   "source": "",
   "id": "82560e4d0ff7ea1a",
   "execution_count": 13,
   "outputs": []
  },
  {
   "metadata": {},
   "cell_type": "code",
   "execution_count": null,
   "source": "",
   "id": "599454c1be7956cb",
   "outputs": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:paddle_demo] *",
   "language": "python",
   "name": "conda-env-paddle_demo-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
